import logging
import os
from trafilatura import fetch_url, extract
import hashlib

article_cache_dir = "cache/articles"

def fetch(url):
    logging.info('Fetching:' + url)

    os.makedirs(article_cache_dir, exist_ok=True)
    url_hash = hashlib.md5(url.encode('utf-8')).hexdigest()
    cache_file = os.path.join(article_cache_dir, f"{url_hash}.md")

    if os.path.exists(cache_file):
        logging.info('Loading from cache...')
        with open(cache_file, 'r', encoding='utf-8') as file:
            return file.read()
    download = fetch_url(url)
    result = extract(download, output_format="markdown", include_comments=True, with_metadata=True)

    with open(cache_file, 'w', encoding='utf-8') as file:
        file.write(result)